package ca.uwindsor.cs.deepweb.utility.gov2;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.LockObtainFailedException;

import ca.uwindsor.cs.deepweb.estimation.DictionariesFilesContentIndexer;
import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;
import ca.uwindsor.cs.deepweb.utility.NekoHTMLHandler;

public class GOV2IndexerwithSizeRange extends DictionariesFilesContentIndexer {

	protected int minsize;

	protected int maxsize;

	private int contentlength;

	public GOV2IndexerwithSizeRange(String sourcepath, String indexdir,
			boolean verbose, int minsize, int maxsize) {
		super(sourcepath, indexdir, "latin1", verbose, true,
				new String[] { "html" });
		// TODO Auto-generated constructor stub
		this.minsize = minsize;
		this.maxsize = maxsize;
	}

	protected void generateIndex(File dir) {
		try {
			File[] dataFiles = dir.listFiles();

			// Add documents to the index
			for (int i = 0; i < dataFiles.length; i++) {
				File file = dataFiles[i];
				if (file.isFile()) {
					if (isValidExtension(file.getName())
							&& file.length() > 2048) {

						NekoHTMLHandler handler = new NekoHTMLHandler(encoding);
						try {
							handler.processContent(new FileInputStream(
									dataFiles[i]));
						} catch (Exception e) {
							// e.printStackTrace();
							System.err.println("Error on parsing: "
									+ dataFiles[i].getCanonicalPath());
							continue;
						}

						String content = handler.getContent().replaceAll(
								"\\s+", " ");
						contentlength = content.length();
						if (contentlength < 2048 || contentlength < minsize
								|| contentlength > maxsize) {
							continue;
						}

						if (isVerbose) {
							System.out.println("Indexing file "
									+ dataFiles[i].getCanonicalPath());
						}

						Document document = new Document();
						// make the ID from the path shorter - get the file name
						String filepath = file.getCanonicalPath();
						int offset1 = filepath
								.lastIndexOf(java.io.File.separator);
						int offset2 = filepath.lastIndexOf(".");
						String word = new String(filepath.substring(
								offset1 + 1, offset2));

						// String word = String.valueOf(globalID);
						// System.out.println(id);

						Field FieldPath = new Field(
								FilesContentIndexer.FIELD_ID, word,
								Field.Store.YES, Field.Index.UN_TOKENIZED);

						Field FieldTitle = new Field(
								FilesContentIndexer.FIELD_TITLE, handler
										.getTitle(), Field.Store.YES,
								Field.Index.UN_TOKENIZED);

						Field FieldBody = new Field(
								FilesContentIndexer.FIELD_CONTENT, content,
								Field.Store.NO, Field.Index.TOKENIZED,
								Field.TermVector.YES);

						document.add(FieldPath);
						document.add(FieldTitle);
						document.add(FieldBody);
						indexWriter.addDocument(document);
						// txtReader.close();
						globalID++;
					}
				} else if (file.isDirectory()) {
					generateIndex(dataFiles[i]);
				}
			}
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		}
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// String[] ext = { "html" };
		// HTMLFilesContentIndexer indexer = new HTMLFilesContentIndexer(
		// "G:/gov2-corpus-extracted", "F:/indexes/GOV2", "UTF-8", true,
		// true, ext);
		// indexer.generateIndex();

		 args = new String[5];
		 args[0] = "--SourceDir=G:\\gov2-corpus-extracted\\";
		 args[1] = "--IndexDir=F:\\indexes\\GOV2_size_4k_5k";
		 args[2] = "-verbose";
		 args[3] = "--minsize=4000";
		 args[4] = "--maxsize=5000";

		String SourceDir = new String();
		String IndexDir = new String();
		boolean isVerbose = false;
		int maxsize=2048;
		int minsize=2048;

		if (args.length < 4) {
			System.out
					.println("usage: java -jar GOV2IndexerwithSizeRange --SourceDir=sourcefiledirectory --IndexDir=indexfiledirectory [-verbose] (--minsize=) (--maxsize=) !");
			System.exit(0);
		} else {
			for (int i = 0; i < args.length; i++) {
				if (args[i].startsWith("--SourceDir=")) {
					SourceDir = args[i].substring(12);
				} else if (args[i].startsWith("--IndexDir=")) {
					IndexDir = args[i].substring(11);
				} else if (args[i].equals("-verbose")) {
					isVerbose = true;
				} else if(args[i].startsWith("--minsize=")){
					minsize = Integer.parseInt(args[i].substring(10));
				} else if(args[i].startsWith("--maxsize=")){
					maxsize = Integer.parseInt(args[i].substring(10));
				}
			}
		}

		if (SourceDir.equals("") || IndexDir.equals("")) {
			System.out
					.println("usage: java -jar GOV2IndexerwithSizeRange SourceDir=sourcefiledirectory IndexDir=indexfiledirectory [-verbose] (--minsize=) (--maxsize=) !");
			System.exit(0);
		} else {
			GOV2IndexerwithSizeRange indexer;
			// if (isRecursive) {
			// indexer = new HTMLFilesContentIndexer(SourceDir, IndexDir,
			// encoding,
			// isVerbose, isCreate, ext);
			// } else {
			// indexer = new HTMLFilesContentIndexer (SourceDir, IndexDir,
			// encoding,
			// isVerbose, isCreate, ext);
			// }
			indexer = new GOV2IndexerwithSizeRange(SourceDir, IndexDir,
					isVerbose, minsize ,maxsize );
			indexer.generateIndex();
		}

	}

}
